<!DOCTYPE html>
<html>
 <head>
  <meta charset="utf-8"/>
  <meta content="width=device-width, initial-scale=1.0" name="viewport"/>
  <meta content="width=device-width,initial-scale=1" name="viewport"/>
  <meta content="ie=edge" http-equiv="x-ua-compatible"/>
  <meta content="Copy to clipboard" name="lang:clipboard.copy"/>
  <meta content="Copied to clipboard" name="lang:clipboard.copied"/>
  <meta content="en" name="lang:search.language"/>
  <meta content="True" name="lang:search.pipeline.stopwords"/>
  <meta content="True" name="lang:search.pipeline.trimmer"/>
  <meta content="No matching documents" name="lang:search.result.none"/>
  <meta content="1 matching document" name="lang:search.result.one"/>
  <meta content="# matching documents" name="lang:search.result.other"/>
  <meta content="[\s\-]+" name="lang:search.tokenizer"/>
  <link crossorigin="" href="https://fonts.gstatic.com/" rel="preconnect"/>
  <link href="https://fonts.googleapis.com/css?family=Roboto+Mono:400,500,700|Roboto:300,400,400i,700&amp;display=fallback" rel="stylesheet"/>
  <style>
   body,
      input {
        font-family: "Roboto", "Helvetica Neue", Helvetica, Arial, sans-serif
      }

      code,
      kbd,
      pre {
        font-family: "Roboto Mono", "Courier New", Courier, monospace
      }
  </style>
  <link href="../_static/stylesheets/application.css" rel="stylesheet"/>
  <link href="../_static/stylesheets/application-palette.css" rel="stylesheet"/>
  <link href="../_static/stylesheets/application-fixes.css" rel="stylesheet"/>
  <link href="../_static/fonts/material-icons.css" rel="stylesheet"/>
  <meta content="84bd00" name="theme-color"/>
  <script src="../_static/javascripts/modernizr.js">
  </script>
  <title>
   TRTorch Getting Started - LeNet — TRTorch v0.1.0 documentation
  </title>
  <link href="../_static/material.css" rel="stylesheet" type="text/css"/>
  <link href="../_static/pygments.css" rel="stylesheet" type="text/css"/>
  <link href="../_static/collapsible-lists/css/tree_view.css" rel="stylesheet" type="text/css"/>
  <script data-url_root="../" id="documentation_options" src="../_static/documentation_options.js">
  </script>
  <script src="../_static/jquery.js">
  </script>
  <script src="../_static/underscore.js">
  </script>
  <script src="../_static/doctools.js">
  </script>
  <script src="../_static/language_data.js">
  </script>
  <script src="../_static/collapsible-lists/js/CollapsibleLists.compressed.js">
  </script>
  <script src="../_static/collapsible-lists/js/apply-collapsible-lists.js">
  </script>
  <script crossorigin="anonymous" integrity="sha256-Ae2Vz/4ePdIu6ZyI/5ZGsYnb+m0JlOmKPjt6XZ9JJkA=" src="https://cdnjs.cloudflare.com/ajax/libs/require.js/2.3.4/require.min.js">
  </script>
  <link href="../genindex.html" rel="index" title="Index"/>
  <link href="../search.html" rel="search" title="Search"/>
  <link href="ssd-object-detection-demo.html" rel="next" title="Object Detection with TRTorch (SSD)"/>
  <link href="../tutorials/use_from_pytorch.html" rel="prev" title="Using TRTorch Directly From PyTorch"/>
 </head>
 <body data-md-color-accent="light-green" data-md-color-primary="light-green" dir="ltr">
  <svg class="md-svg">
   <defs data-children-count="0">
    <svg height="448" id="__github" viewbox="0 0 416 448" width="416" xmlns="http://www.w3.org/2000/svg">
     <path d="M160 304q0 10-3.125 20.5t-10.75 19T128 352t-18.125-8.5-10.75-19T96 304t3.125-20.5 10.75-19T128 256t18.125 8.5 10.75 19T160 304zm160 0q0 10-3.125 20.5t-10.75 19T288 352t-18.125-8.5-10.75-19T256 304t3.125-20.5 10.75-19T288 256t18.125 8.5 10.75 19T320 304zm40 0q0-30-17.25-51T296 232q-10.25 0-48.75 5.25Q229.5 240 208 240t-39.25-2.75Q130.75 232 120 232q-29.5 0-46.75 21T56 304q0 22 8 38.375t20.25 25.75 30.5 15 35 7.375 37.25 1.75h42q20.5 0 37.25-1.75t35-7.375 30.5-15 20.25-25.75T360 304zm56-44q0 51.75-15.25 82.75-9.5 19.25-26.375 33.25t-35.25 21.5-42.5 11.875-42.875 5.5T212 416q-19.5 0-35.5-.75t-36.875-3.125-38.125-7.5-34.25-12.875T37 371.5t-21.5-28.75Q0 312 0 260q0-59.25 34-99-6.75-20.5-6.75-42.5 0-29 12.75-54.5 27 0 47.5 9.875t47.25 30.875Q171.5 96 212 96q37 0 70 8 26.25-20.5 46.75-30.25T376 64q12.75 25.5 12.75 54.5 0 21.75-6.75 42 34 40 34 99.5z" fill="currentColor">
     </path>
    </svg>
   </defs>
  </svg>
  <input class="md-toggle" data-md-toggle="drawer" id="__drawer" type="checkbox"/>
  <input class="md-toggle" data-md-toggle="search" id="__search" type="checkbox"/>
  <label class="md-overlay" data-md-component="overlay" for="__drawer">
  </label>
  <a class="md-skip" href="#_notebooks/lenet-getting-started" tabindex="1">
   Skip to content
  </a>
  <header class="md-header" data-md-component="header">
   <nav class="md-header-nav md-grid">
    <div class="md-flex navheader">
     <div class="md-flex__cell md-flex__cell--shrink">
      <a class="md-header-nav__button md-logo" href="../index.html" title="TRTorch v0.1.0 documentation">
       <i class="md-icon">
        
       </i>
      </a>
     </div>
     <div class="md-flex__cell md-flex__cell--shrink">
      <label class="md-icon md-icon--menu md-header-nav__button" for="__drawer">
      </label>
     </div>
     <div class="md-flex__cell md-flex__cell--stretch">
      <div class="md-flex__ellipsis md-header-nav__title" data-md-component="title">
       <span class="md-header-nav__topic">
        TRTorch
       </span>
       <span class="md-header-nav__topic">
        TRTorch Getting Started - LeNet
       </span>
      </div>
     </div>
     <div class="md-flex__cell md-flex__cell--shrink">
      <label class="md-icon md-icon--search md-header-nav__button" for="__search">
      </label>
      <div class="md-search" data-md-component="search" role="dialog">
       <label class="md-search__overlay" for="__search">
       </label>
       <div class="md-search__inner" role="search">
        <form action="../search.html" class="md-search__form" method="GET" name="search">
         <input autocapitalize="off" autocomplete="off" class="md-search__input" data-md-component="query" data-md-state="active" name="q" placeholder="Search" spellcheck="false" type="text"/>
         <label class="md-icon md-search__icon" for="__search">
         </label>
         <button class="md-icon md-search__icon" data-md-component="reset" tabindex="-1" type="reset">
          
         </button>
        </form>
        <div class="md-search__output">
         <div class="md-search__scrollwrap" data-md-scrollfix="">
          <div class="md-search-result" data-md-component="result">
           <div class="md-search-result__meta">
            Type to start searching
           </div>
           <ol class="md-search-result__list">
           </ol>
          </div>
         </div>
        </div>
       </div>
      </div>
     </div>
     <div class="md-flex__cell md-flex__cell--shrink">
      <div class="md-header-nav__source">
       <a class="md-source" data-md-source="github" href="https://github.com/nvidia/TRTorch/" title="Go to repository">
        <div class="md-source__icon">
         <svg height="28" viewbox="0 0 24 24" width="28" xmlns="http://www.w3.org/2000/svg" xmlns:xlink="http://www.w3.org/1999/xlink">
          <use height="24" width="24" xlink:href="#__github">
          </use>
         </svg>
        </div>
        <div class="md-source__repository">
         TRTorch
        </div>
       </a>
      </div>
     </div>
     <div class="md-flex__cell md-flex__cell--shrink dropdown">
      <button class="dropdownbutton">
       Versions
      </button>
      <div class="dropdown-content md-hero">
       <a href="https://nvidia.github.io/TRTorch/" title="master">
        master
       </a>
       <a href="https://nvidia.github.io/TRTorch/v0.1.0/" title="v0.1.0">
        v0.1.0
       </a>
       <a href="https://nvidia.github.io/TRTorch/v0.0.3/" title="v0.0.3">
        v0.0.3
       </a>
       <a href="https://nvidia.github.io/TRTorch/v0.0.2/" title="v0.0.2">
        v0.0.2
       </a>
       <a href="https://nvidia.github.io/TRTorch/v0.0.1/" title="v0.0.1">
        v0.0.1
       </a>
      </div>
     </div>
    </div>
   </nav>
  </header>
  <div class="md-container">
   <nav class="md-tabs" data-md-component="tabs">
    <div class="md-tabs__inner md-grid">
     <ul class="md-tabs__list">
      <li class="md-tabs__item">
       <a class="md-tabs__link" href="../index.html">
        TRTorch v0.1.0 documentation
       </a>
      </li>
     </ul>
    </div>
   </nav>
   <main class="md-main">
    <div class="md-main__inner md-grid" data-md-component="container">
     <div class="md-sidebar md-sidebar--primary" data-md-component="navigation">
      <div class="md-sidebar__scrollwrap">
       <div class="md-sidebar__inner">
        <nav class="md-nav md-nav--primary" data-md-level="0">
         <label class="md-nav__title md-nav__title--site" for="__drawer">
          <a class="md-nav__button md-logo" href="../index.html" title="TRTorch v0.1.0 documentation">
           <i class="md-icon">
            
           </i>
          </a>
          <a href="../index.html" title="TRTorch v0.1.0 documentation">
           TRTorch
          </a>
         </label>
         <div class="md-nav__source">
          <a class="md-source" data-md-source="github" href="https://github.com/nvidia/TRTorch/" title="Go to repository">
           <div class="md-source__icon">
            <svg height="28" viewbox="0 0 24 24" width="28" xmlns="http://www.w3.org/2000/svg" xmlns:xlink="http://www.w3.org/1999/xlink">
             <use height="24" width="24" xlink:href="#__github">
             </use>
            </svg>
           </div>
           <div class="md-source__repository">
            TRTorch
           </div>
          </a>
         </div>
         <ul class="md-nav__list">
          <li class="md-nav__item">
           <span class="md-nav__link caption">
            <span class="caption-text">
             Getting Started
            </span>
           </span>
          </li>
          <li class="md-nav__item">
           <a class="md-nav__link" href="../tutorials/installation.html">
            Installation
           </a>
          </li>
          <li class="md-nav__item">
           <a class="md-nav__link" href="../tutorials/getting_started.html">
            Getting Started
           </a>
          </li>
          <li class="md-nav__item">
           <a class="md-nav__link" href="../tutorials/ptq.html">
            Post Training Quantization (PTQ)
           </a>
          </li>
          <li class="md-nav__item">
           <a class="md-nav__link" href="../tutorials/trtorchc.html">
            trtorchc
           </a>
          </li>
          <li class="md-nav__item">
           <a class="md-nav__link" href="../tutorials/use_from_pytorch.html">
            Using TRTorch Directly From PyTorch
           </a>
          </li>
          <li class="md-nav__item">
           <span class="md-nav__link caption">
            <span class="caption-text">
             Notebooks
            </span>
           </span>
          </li>
          <li class="md-nav__item">
           <input class="md-toggle md-nav__toggle" data-md-toggle="toc" id="__toc" type="checkbox"/>
           <label class="md-nav__link md-nav__link--active" for="__toc">
            TRTorch Getting Started - LeNet
           </label>
           <a class="md-nav__link md-nav__link--active" href="#">
            TRTorch Getting Started - LeNet
           </a>
           <nav class="md-nav md-nav--secondary">
            <label class="md-nav__title" for="__toc">
             Contents
            </label>
            <ul class="md-nav__list" data-md-scrollfix="">
             <li class="md-nav__item">
              <a class="md-nav__link" href="#notebooks-lenet-getting-started--page-root">
               TRTorch Getting Started - LeNet
              </a>
              <nav class="md-nav">
               <ul class="md-nav__list">
                <li class="md-nav__item">
                 <a class="md-nav__link" href="#Overview">
                  Overview
                 </a>
                 <nav class="md-nav">
                  <ul class="md-nav__list">
                   <li class="md-nav__item">
                    <a class="md-nav__link" href="#Learning-objectives">
                     Learning objectives
                    </a>
                   </li>
                  </ul>
                 </nav>
                </li>
                <li class="md-nav__item">
                 <a class="md-nav__link" href="#Content">
                  Content
                 </a>
                 <nav class="md-nav">
                  <ul class="md-nav__list">
                   <li class="md-nav__item">
                    <a class="md-nav__link" href="#PyTorch-model">
                     PyTorch model
                    </a>
                   </li>
                   <li class="md-nav__item">
                    <a class="md-nav__link" href="#Tracing">
                     Tracing
                    </a>
                   </li>
                   <li class="md-nav__item">
                    <a class="md-nav__link" href="#Scripting">
                     Scripting
                    </a>
                   </li>
                   <li class="md-nav__item">
                    <a class="md-nav__link" href="#TorchScript-traced-model">
                     TorchScript traced model
                    </a>
                   </li>
                   <li class="md-nav__item">
                    <a class="md-nav__link" href="#TorchScript-script-model">
                     TorchScript script model
                    </a>
                   </li>
                  </ul>
                 </nav>
                </li>
                <li class="md-nav__item">
                 <a class="md-nav__link" href="#Conclusion">
                  Conclusion
                 </a>
                 <nav class="md-nav">
                  <ul class="md-nav__list">
                   <li class="md-nav__item">
                    <a class="md-nav__link" href="#What’s-next">
                     What’s next
                    </a>
                   </li>
                  </ul>
                 </nav>
                </li>
               </ul>
              </nav>
             </li>
             <li class="md-nav__item">
              <a class="md-nav__extra_link" href="../_sources/_notebooks/lenet-getting-started.ipynb.txt">
               Show Source
              </a>
             </li>
            </ul>
           </nav>
          </li>
          <li class="md-nav__item">
           <a class="md-nav__link" href="ssd-object-detection-demo.html">
            Object Detection with TRTorch (SSD)
           </a>
          </li>
          <li class="md-nav__item">
           <span class="md-nav__link caption">
            <span class="caption-text">
             Python API Documenation
            </span>
           </span>
          </li>
          <li class="md-nav__item">
           <a class="md-nav__link" href="../py_api/trtorch.html">
            trtorch
           </a>
          </li>
          <li class="md-nav__item">
           <a class="md-nav__link" href="../py_api/logging.html">
            trtorch.logging
           </a>
          </li>
          <li class="md-nav__item">
           <span class="md-nav__link caption">
            <span class="caption-text">
             C++ API Documenation
            </span>
           </span>
          </li>
          <li class="md-nav__item">
           <a class="md-nav__link" href="../_cpp_api/trtorch_cpp.html">
            TRTorch C++ API
           </a>
          </li>
          <li class="md-nav__item">
           <span class="md-nav__link caption">
            <span class="caption-text">
             Contributor Documentation
            </span>
           </span>
          </li>
          <li class="md-nav__item">
           <a class="md-nav__link" href="../contributors/system_overview.html">
            System Overview
           </a>
          </li>
          <li class="md-nav__item">
           <a class="md-nav__link" href="../contributors/writing_converters.html">
            Writing Converters
           </a>
          </li>
          <li class="md-nav__item">
           <a class="md-nav__link" href="../contributors/useful_links.html">
            Useful Links for TRTorch Development
           </a>
          </li>
         </ul>
        </nav>
       </div>
      </div>
     </div>
     <div class="md-sidebar md-sidebar--secondary" data-md-component="toc">
      <div class="md-sidebar__scrollwrap">
       <div class="md-sidebar__inner">
        <nav class="md-nav md-nav--secondary">
         <label class="md-nav__title" for="__toc">
          Contents
         </label>
         <ul class="md-nav__list" data-md-scrollfix="">
          <li class="md-nav__item">
           <a class="md-nav__link" href="#notebooks-lenet-getting-started--page-root">
            TRTorch Getting Started - LeNet
           </a>
           <nav class="md-nav">
            <ul class="md-nav__list">
             <li class="md-nav__item">
              <a class="md-nav__link" href="#Overview">
               Overview
              </a>
              <nav class="md-nav">
               <ul class="md-nav__list">
                <li class="md-nav__item">
                 <a class="md-nav__link" href="#Learning-objectives">
                  Learning objectives
                 </a>
                </li>
               </ul>
              </nav>
             </li>
             <li class="md-nav__item">
              <a class="md-nav__link" href="#Content">
               Content
              </a>
              <nav class="md-nav">
               <ul class="md-nav__list">
                <li class="md-nav__item">
                 <a class="md-nav__link" href="#PyTorch-model">
                  PyTorch model
                 </a>
                </li>
                <li class="md-nav__item">
                 <a class="md-nav__link" href="#Tracing">
                  Tracing
                 </a>
                </li>
                <li class="md-nav__item">
                 <a class="md-nav__link" href="#Scripting">
                  Scripting
                 </a>
                </li>
                <li class="md-nav__item">
                 <a class="md-nav__link" href="#TorchScript-traced-model">
                  TorchScript traced model
                 </a>
                </li>
                <li class="md-nav__item">
                 <a class="md-nav__link" href="#TorchScript-script-model">
                  TorchScript script model
                 </a>
                </li>
               </ul>
              </nav>
             </li>
             <li class="md-nav__item">
              <a class="md-nav__link" href="#Conclusion">
               Conclusion
              </a>
              <nav class="md-nav">
               <ul class="md-nav__list">
                <li class="md-nav__item">
                 <a class="md-nav__link" href="#What’s-next">
                  What’s next
                 </a>
                </li>
               </ul>
              </nav>
             </li>
            </ul>
           </nav>
          </li>
          <li class="md-nav__item">
           <a class="md-nav__extra_link" href="../_sources/_notebooks/lenet-getting-started.ipynb.txt">
            Show Source
           </a>
          </li>
          <li class="md-nav__item" id="searchbox">
          </li>
         </ul>
        </nav>
       </div>
      </div>
     </div>
     <div class="md-content">
      <article class="md-content__inner md-typeset" role="main">
       <style>
        /* CSS for nbsphinx extension */

/* remove conflicting styling from Sphinx themes */
div.nbinput.container,
div.nbinput.container div.prompt,
div.nbinput.container div.input_area,
div.nbinput.container div[class*=highlight],
div.nbinput.container div[class*=highlight] pre,
div.nboutput.container,
div.nboutput.container div.prompt,
div.nboutput.container div.output_area,
div.nboutput.container div[class*=highlight],
div.nboutput.container div[class*=highlight] pre {
    background: none;
    border: none;
    padding: 0 0;
    margin: 0;
    box-shadow: none;
}

/* avoid gaps between output lines */
div.nboutput.container div[class*=highlight] pre {
    line-height: normal;
}

/* input/output containers */
div.nbinput.container,
div.nboutput.container {
    display: -webkit-flex;
    display: flex;
    align-items: flex-start;
    margin: 0;
    width: 100%;
}
@media (max-width: 540px) {
    div.nbinput.container,
    div.nboutput.container {
        flex-direction: column;
    }
}

/* input container */
div.nbinput.container {
    padding-top: 5px;
}

/* last container */
div.nblast.container {
    padding-bottom: 5px;
}

/* input prompt */
div.nbinput.container div.prompt pre {
    color: #307FC1;
}

/* output prompt */
div.nboutput.container div.prompt pre {
    color: #BF5B3D;
}

/* all prompts */
div.nbinput.container div.prompt,
div.nboutput.container div.prompt {
    width: 4.5ex;
    padding-top: 5px;
    position: relative;
    user-select: none;
}

div.nbinput.container div.prompt > div,
div.nboutput.container div.prompt > div {
    position: absolute;
    right: 0;
    margin-right: 0.3ex;
}

@media (max-width: 540px) {
    div.nbinput.container div.prompt,
    div.nboutput.container div.prompt {
        width: unset;
        text-align: left;
        padding: 0.4em;
    }
    div.nboutput.container div.prompt.empty {
        padding: 0;
    }

    div.nbinput.container div.prompt > div,
    div.nboutput.container div.prompt > div {
        position: unset;
    }
}

/* disable scrollbars on prompts */
div.nbinput.container div.prompt pre,
div.nboutput.container div.prompt pre {
    overflow: hidden;
}

/* input/output area */
div.nbinput.container div.input_area,
div.nboutput.container div.output_area {
    -webkit-flex: 1;
    flex: 1;
    overflow: auto;
}
@media (max-width: 540px) {
    div.nbinput.container div.input_area,
    div.nboutput.container div.output_area {
        width: 100%;
    }
}

/* input area */
div.nbinput.container div.input_area {
    border: 1px solid #e0e0e0;
    border-radius: 2px;
    background: #f5f5f5;
}

/* override MathJax center alignment in output cells */
div.nboutput.container div[class*=MathJax] {
    text-align: left !important;
}

/* override sphinx.ext.imgmath center alignment in output cells */
div.nboutput.container div.math p {
    text-align: left;
}

/* standard error */
div.nboutput.container div.output_area.stderr {
    background: #fdd;
}

/* ANSI colors */
.ansi-black-fg { color: #3E424D; }
.ansi-black-bg { background-color: #3E424D; }
.ansi-black-intense-fg { color: #282C36; }
.ansi-black-intense-bg { background-color: #282C36; }
.ansi-red-fg { color: #E75C58; }
.ansi-red-bg { background-color: #E75C58; }
.ansi-red-intense-fg { color: #B22B31; }
.ansi-red-intense-bg { background-color: #B22B31; }
.ansi-green-fg { color: #00A250; }
.ansi-green-bg { background-color: #00A250; }
.ansi-green-intense-fg { color: #007427; }
.ansi-green-intense-bg { background-color: #007427; }
.ansi-yellow-fg { color: #DDB62B; }
.ansi-yellow-bg { background-color: #DDB62B; }
.ansi-yellow-intense-fg { color: #B27D12; }
.ansi-yellow-intense-bg { background-color: #B27D12; }
.ansi-blue-fg { color: #208FFB; }
.ansi-blue-bg { background-color: #208FFB; }
.ansi-blue-intense-fg { color: #0065CA; }
.ansi-blue-intense-bg { background-color: #0065CA; }
.ansi-magenta-fg { color: #D160C4; }
.ansi-magenta-bg { background-color: #D160C4; }
.ansi-magenta-intense-fg { color: #A03196; }
.ansi-magenta-intense-bg { background-color: #A03196; }
.ansi-cyan-fg { color: #60C6C8; }
.ansi-cyan-bg { background-color: #60C6C8; }
.ansi-cyan-intense-fg { color: #258F8F; }
.ansi-cyan-intense-bg { background-color: #258F8F; }
.ansi-white-fg { color: #C5C1B4; }
.ansi-white-bg { background-color: #C5C1B4; }
.ansi-white-intense-fg { color: #A1A6B2; }
.ansi-white-intense-bg { background-color: #A1A6B2; }

.ansi-default-inverse-fg { color: #FFFFFF; }
.ansi-default-inverse-bg { background-color: #000000; }

.ansi-bold { font-weight: bold; }
.ansi-underline { text-decoration: underline; }


div.nbinput.container div.input_area div[class*=highlight] > pre,
div.nboutput.container div.output_area div[class*=highlight] > pre,
div.nboutput.container div.output_area div[class*=highlight].math,
div.nboutput.container div.output_area.rendered_html,
div.nboutput.container div.output_area > div.output_javascript,
div.nboutput.container div.output_area:not(.rendered_html) > img{
    padding: 5px;
}

/* fix copybtn overflow problem in chromium (needed for 'sphinx_copybutton') */
div.nbinput.container div.input_area > div[class^='highlight'],
div.nboutput.container div.output_area > div[class^='highlight']{
    overflow-y: hidden;
}

/* hide copybtn icon on prompts (needed for 'sphinx_copybutton') */
.prompt a.copybtn {
    display: none;
}

/* Some additional styling taken form the Jupyter notebook CSS */
div.rendered_html table {
  border: none;
  border-collapse: collapse;
  border-spacing: 0;
  color: black;
  font-size: 12px;
  table-layout: fixed;
}
div.rendered_html thead {
  border-bottom: 1px solid black;
  vertical-align: bottom;
}
div.rendered_html tr,
div.rendered_html th,
div.rendered_html td {
  text-align: right;
  vertical-align: middle;
  padding: 0.5em 0.5em;
  line-height: normal;
  white-space: normal;
  max-width: none;
  border: none;
}
div.rendered_html th {
  font-weight: bold;
}
div.rendered_html tbody tr:nth-child(odd) {
  background: #f5f5f5;
}
div.rendered_html tbody tr:hover {
  background: rgba(66, 165, 245, 0.2);
}
       </style>
       <div class="nbinput nblast docutils container">
        <div class="prompt highlight-none notranslate">
         <div class="highlight">
          <pre><span></span>[1]:
</pre>
         </div>
        </div>
        <div class="input_area highlight-ipython3 notranslate">
         <div class="highlight">
          <pre>
<span></span><span class="c1"># Copyright 2019 NVIDIA Corporation. All Rights Reserved.</span>
<span class="c1">#</span>
<span class="c1"># Licensed under the Apache License, Version 2.0 (the "License");</span>
<span class="c1"># you may not use this file except in compliance with the License.</span>
<span class="c1"># You may obtain a copy of the License at</span>
<span class="c1">#</span>
<span class="c1">#     http://www.apache.org/licenses/LICENSE-2.0</span>
<span class="c1">#</span>
<span class="c1"># Unless required by applicable law or agreed to in writing, software</span>
<span class="c1"># distributed under the License is distributed on an "AS IS" BASIS,</span>
<span class="c1"># WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.</span>
<span class="c1"># See the License for the specific language governing permissions and</span>
<span class="c1"># limitations under the License.</span>
<span class="c1"># ==============================================================================</span>
</pre>
         </div>
        </div>
       </div>
       <p>
        <img alt="4138f9f08e5444c69add3d78c932799d" src="http://developer.download.nvidia.com/compute/machine-learning/frameworks/nvidia_logo.png"/>
       </p>
       <h1 id="notebooks-lenet-getting-started--page-root">
        TRTorch Getting Started - LeNet
        <a class="headerlink" href="#notebooks-lenet-getting-started--page-root" title="Permalink to this headline">
         ¶
        </a>
       </h1>
       <h2 id="Overview">
        Overview
        <a class="headerlink" href="#Overview" title="Permalink to this headline">
         ¶
        </a>
       </h2>
       <p>
        In the practice of developing machine learning models, there are few tools as approachable as PyTorch for developing and experimenting in designing machine learning models. The power of PyTorch comes from its deep integration into Python, its flexibility and its approach to automatic differentiation and execution (eager execution). However, when moving from research into production, the requirements change and we may no longer want that deep Python integration and we want optimization to get the
best performance we can on our deployment platform. In PyTorch 1.0, TorchScript was introduced as a method to separate your PyTorch model from Python, make it portable and optimizable. TorchScript uses PyTorch’s JIT compiler to transform your normal PyTorch code which gets interpreted by the Python interpreter to an intermediate representation (IR) which can have optimizations run on it and at runtime can get interpreted by the PyTorch JIT interpreter. For PyTorch this has opened up a whole new
world of possibilities, including deployment in other languages like C++. It also introduces a structured graph based format that we can use to do down to the kernel level optimization of models for inference.
       </p>
       <p>
        When deploying on NVIDIA GPUs TensorRT, NVIDIA’s Deep Learning Optimization SDK and Runtime is able to take models from any major framework and specifically tune them to perform better on specific target hardware in the NVIDIA family be it an A100, TITAN V, Jetson Xavier or NVIDIA’s Deep Learning Accelerator. TensorRT performs a couple sets of optimizations to achieve this. TensorRT fuses layers and tensors in the model graph, it then uses a large kernel library to select implementations that
perform best on the target GPU. TensorRT also has strong support for reduced operating precision execution which allows users to leverage the Tensor Cores on Volta and newer GPUs as well as reducing memory and computation footprints on device.
       </p>
       <p>
        TRTorch is a compiler that uses TensorRT to optimize TorchScript code, compiling standard TorchScript modules into ones that internally run with TensorRT optimizations. This enables you to continue to remain in the PyTorch ecosystem, using all the great features PyTorch has such as module composability, its flexible tensor implementation, data loaders and more. TRTorch is available to use with both PyTorch and LibTorch.
       </p>
       <h3 id="Learning-objectives">
        Learning objectives
        <a class="headerlink" href="#Learning-objectives" title="Permalink to this headline">
         ¶
        </a>
       </h3>
       <p>
        This notebook demonstrates the steps for compiling a TorchScript module with TRTorch on a simple LeNet network.
       </p>
       <h2 id="Content">
        Content
        <a class="headerlink" href="#Content" title="Permalink to this headline">
         ¶
        </a>
       </h2>
       <ol class="arabic simple">
        <li>
         <p>
          <a class="reference external" href="#1">
           Requirements
          </a>
         </p>
        </li>
        <li>
         <p>
          <a class="reference external" href="#2">
           Creating TorchScript modules
          </a>
         </p>
        </li>
        <li>
         <p>
          <a class="reference external" href="#3">
           Compiling with TRTorch
          </a>
         </p>
        </li>
       </ol>
       <p>
        ## 1. Requirements
       </p>
       <p>
        Follow the steps in
        <code class="docutils literal notranslate">
         <span class="pre">
          notebooks/README
         </span>
        </code>
        to prepare a Docker container, within which you can run this notebook.
       </p>
       <p>
        ## 2. Creating TorchScript modules
       </p>
       <p>
        Here we create two submodules for a feature extractor and a classifier and stitch them together in a single LeNet module. In this case this is overkill but modules give us granular control over our program including where we decide to optimize and where we don’t. It is also the unit that the TorchScript compiler operates on. So you can decide to only convert/optimize the feature extractor and leave the classifier in standard PyTorch or you can convert the whole thing. When compiling your module
to TorchScript, there are two paths: Tracing and Scripting.
       </p>
       <div class="nbinput nblast docutils container">
        <div class="prompt highlight-none notranslate">
         <div class="highlight">
          <pre><span></span>[2]:
</pre>
         </div>
        </div>
        <div class="input_area highlight-ipython3 notranslate">
         <div class="highlight">
          <pre>
<span></span><span class="kn">import</span> <span class="nn">torch</span>
<span class="kn">from</span> <span class="nn">torch</span> <span class="kn">import</span> <span class="n">nn</span>
<span class="kn">import</span> <span class="nn">torch.nn.functional</span> <span class="k">as</span> <span class="nn">F</span>

<span class="k">class</span> <span class="nc">LeNetFeatExtractor</span><span class="p">(</span><span class="n">nn</span><span class="o">.</span><span class="n">Module</span><span class="p">):</span>
    <span class="k">def</span> <span class="fm">__init__</span><span class="p">(</span><span class="bp">self</span><span class="p">):</span>
        <span class="nb">super</span><span class="p">(</span><span class="n">LeNetFeatExtractor</span><span class="p">,</span> <span class="bp">self</span><span class="p">)</span><span class="o">.</span><span class="fm">__init__</span><span class="p">()</span>
        <span class="bp">self</span><span class="o">.</span><span class="n">conv1</span> <span class="o">=</span> <span class="n">nn</span><span class="o">.</span><span class="n">Conv2d</span><span class="p">(</span><span class="mi">1</span><span class="p">,</span> <span class="mi">6</span><span class="p">,</span> <span class="mi">3</span><span class="p">)</span>
        <span class="bp">self</span><span class="o">.</span><span class="n">conv2</span> <span class="o">=</span> <span class="n">nn</span><span class="o">.</span><span class="n">Conv2d</span><span class="p">(</span><span class="mi">6</span><span class="p">,</span> <span class="mi">16</span><span class="p">,</span> <span class="mi">3</span><span class="p">)</span>

    <span class="k">def</span> <span class="nf">forward</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span> <span class="n">x</span><span class="p">):</span>
        <span class="n">x</span> <span class="o">=</span> <span class="n">F</span><span class="o">.</span><span class="n">max_pool2d</span><span class="p">(</span><span class="n">F</span><span class="o">.</span><span class="n">relu</span><span class="p">(</span><span class="bp">self</span><span class="o">.</span><span class="n">conv1</span><span class="p">(</span><span class="n">x</span><span class="p">)),</span> <span class="p">(</span><span class="mi">2</span><span class="p">,</span> <span class="mi">2</span><span class="p">))</span>
        <span class="n">x</span> <span class="o">=</span> <span class="n">F</span><span class="o">.</span><span class="n">max_pool2d</span><span class="p">(</span><span class="n">F</span><span class="o">.</span><span class="n">relu</span><span class="p">(</span><span class="bp">self</span><span class="o">.</span><span class="n">conv2</span><span class="p">(</span><span class="n">x</span><span class="p">)),</span> <span class="mi">2</span><span class="p">)</span>
        <span class="k">return</span> <span class="n">x</span>

<span class="k">class</span> <span class="nc">LeNetClassifier</span><span class="p">(</span><span class="n">nn</span><span class="o">.</span><span class="n">Module</span><span class="p">):</span>
    <span class="k">def</span> <span class="fm">__init__</span><span class="p">(</span><span class="bp">self</span><span class="p">):</span>
        <span class="nb">super</span><span class="p">(</span><span class="n">LeNetClassifier</span><span class="p">,</span> <span class="bp">self</span><span class="p">)</span><span class="o">.</span><span class="fm">__init__</span><span class="p">()</span>
        <span class="bp">self</span><span class="o">.</span><span class="n">fc1</span> <span class="o">=</span> <span class="n">nn</span><span class="o">.</span><span class="n">Linear</span><span class="p">(</span><span class="mi">16</span> <span class="o">*</span> <span class="mi">6</span> <span class="o">*</span> <span class="mi">6</span><span class="p">,</span> <span class="mi">120</span><span class="p">)</span>
        <span class="bp">self</span><span class="o">.</span><span class="n">fc2</span> <span class="o">=</span> <span class="n">nn</span><span class="o">.</span><span class="n">Linear</span><span class="p">(</span><span class="mi">120</span><span class="p">,</span> <span class="mi">84</span><span class="p">)</span>
        <span class="bp">self</span><span class="o">.</span><span class="n">fc3</span> <span class="o">=</span> <span class="n">nn</span><span class="o">.</span><span class="n">Linear</span><span class="p">(</span><span class="mi">84</span><span class="p">,</span> <span class="mi">10</span><span class="p">)</span>

    <span class="k">def</span> <span class="nf">forward</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span> <span class="n">x</span><span class="p">):</span>
        <span class="n">x</span> <span class="o">=</span> <span class="n">torch</span><span class="o">.</span><span class="n">flatten</span><span class="p">(</span><span class="n">x</span><span class="p">,</span><span class="mi">1</span><span class="p">)</span>
        <span class="n">x</span> <span class="o">=</span> <span class="n">F</span><span class="o">.</span><span class="n">relu</span><span class="p">(</span><span class="bp">self</span><span class="o">.</span><span class="n">fc1</span><span class="p">(</span><span class="n">x</span><span class="p">))</span>
        <span class="n">x</span> <span class="o">=</span> <span class="n">F</span><span class="o">.</span><span class="n">relu</span><span class="p">(</span><span class="bp">self</span><span class="o">.</span><span class="n">fc2</span><span class="p">(</span><span class="n">x</span><span class="p">))</span>
        <span class="n">x</span> <span class="o">=</span> <span class="bp">self</span><span class="o">.</span><span class="n">fc3</span><span class="p">(</span><span class="n">x</span><span class="p">)</span>
        <span class="k">return</span> <span class="n">x</span>

<span class="k">class</span> <span class="nc">LeNet</span><span class="p">(</span><span class="n">nn</span><span class="o">.</span><span class="n">Module</span><span class="p">):</span>
    <span class="k">def</span> <span class="fm">__init__</span><span class="p">(</span><span class="bp">self</span><span class="p">):</span>
        <span class="nb">super</span><span class="p">(</span><span class="n">LeNet</span><span class="p">,</span> <span class="bp">self</span><span class="p">)</span><span class="o">.</span><span class="fm">__init__</span><span class="p">()</span>
        <span class="bp">self</span><span class="o">.</span><span class="n">feat</span> <span class="o">=</span> <span class="n">LeNetFeatExtractor</span><span class="p">()</span>
        <span class="bp">self</span><span class="o">.</span><span class="n">classifer</span> <span class="o">=</span> <span class="n">LeNetClassifier</span><span class="p">()</span>

    <span class="k">def</span> <span class="nf">forward</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span> <span class="n">x</span><span class="p">):</span>
        <span class="n">x</span> <span class="o">=</span> <span class="bp">self</span><span class="o">.</span><span class="n">feat</span><span class="p">(</span><span class="n">x</span><span class="p">)</span>
        <span class="n">x</span> <span class="o">=</span> <span class="bp">self</span><span class="o">.</span><span class="n">classifer</span><span class="p">(</span><span class="n">x</span><span class="p">)</span>
        <span class="k">return</span> <span class="n">x</span>

</pre>
         </div>
        </div>
       </div>
       <p>
        Let us define a helper function to benchmark a model.
       </p>
       <div class="nbinput nblast docutils container">
        <div class="prompt highlight-none notranslate">
         <div class="highlight">
          <pre><span></span>[3]:
</pre>
         </div>
        </div>
        <div class="input_area highlight-ipython3 notranslate">
         <div class="highlight">
          <pre>
<span></span><span class="kn">import</span> <span class="nn">time</span>
<span class="kn">import</span> <span class="nn">numpy</span> <span class="k">as</span> <span class="nn">np</span>

<span class="kn">import</span> <span class="nn">torch.backends.cudnn</span> <span class="k">as</span> <span class="nn">cudnn</span>
<span class="n">cudnn</span><span class="o">.</span><span class="n">benchmark</span> <span class="o">=</span> <span class="kc">True</span>

<span class="k">def</span> <span class="nf">benchmark</span><span class="p">(</span><span class="n">model</span><span class="p">,</span> <span class="n">input_shape</span><span class="o">=</span><span class="p">(</span><span class="mi">1024</span><span class="p">,</span> <span class="mi">1</span><span class="p">,</span> <span class="mi">32</span><span class="p">,</span> <span class="mi">32</span><span class="p">),</span> <span class="n">dtype</span><span class="o">=</span><span class="s1">'fp32'</span><span class="p">,</span> <span class="n">nwarmup</span><span class="o">=</span><span class="mi">50</span><span class="p">,</span> <span class="n">nruns</span><span class="o">=</span><span class="mi">10000</span><span class="p">):</span>
    <span class="n">input_data</span> <span class="o">=</span> <span class="n">torch</span><span class="o">.</span><span class="n">randn</span><span class="p">(</span><span class="n">input_shape</span><span class="p">)</span>
    <span class="n">input_data</span> <span class="o">=</span> <span class="n">input_data</span><span class="o">.</span><span class="n">to</span><span class="p">(</span><span class="s2">"cuda"</span><span class="p">)</span>
    <span class="k">if</span> <span class="n">dtype</span><span class="o">==</span><span class="s1">'fp16'</span><span class="p">:</span>
        <span class="n">input_data</span> <span class="o">=</span> <span class="n">input_data</span><span class="o">.</span><span class="n">half</span><span class="p">()</span>

    <span class="nb">print</span><span class="p">(</span><span class="s2">"Warm up ..."</span><span class="p">)</span>
    <span class="k">with</span> <span class="n">torch</span><span class="o">.</span><span class="n">no_grad</span><span class="p">():</span>
        <span class="k">for</span> <span class="n">_</span> <span class="ow">in</span> <span class="nb">range</span><span class="p">(</span><span class="n">nwarmup</span><span class="p">):</span>
            <span class="n">features</span> <span class="o">=</span> <span class="n">model</span><span class="p">(</span><span class="n">input_data</span><span class="p">)</span>
    <span class="n">torch</span><span class="o">.</span><span class="n">cuda</span><span class="o">.</span><span class="n">synchronize</span><span class="p">()</span>
    <span class="nb">print</span><span class="p">(</span><span class="s2">"Start timing ..."</span><span class="p">)</span>
    <span class="n">timings</span> <span class="o">=</span> <span class="p">[]</span>
    <span class="k">with</span> <span class="n">torch</span><span class="o">.</span><span class="n">no_grad</span><span class="p">():</span>
        <span class="k">for</span> <span class="n">i</span> <span class="ow">in</span> <span class="nb">range</span><span class="p">(</span><span class="mi">1</span><span class="p">,</span> <span class="n">nruns</span><span class="o">+</span><span class="mi">1</span><span class="p">):</span>
            <span class="n">start_time</span> <span class="o">=</span> <span class="n">time</span><span class="o">.</span><span class="n">time</span><span class="p">()</span>
            <span class="n">features</span> <span class="o">=</span> <span class="n">model</span><span class="p">(</span><span class="n">input_data</span><span class="p">)</span>
            <span class="n">torch</span><span class="o">.</span><span class="n">cuda</span><span class="o">.</span><span class="n">synchronize</span><span class="p">()</span>
            <span class="n">end_time</span> <span class="o">=</span> <span class="n">time</span><span class="o">.</span><span class="n">time</span><span class="p">()</span>
            <span class="n">timings</span><span class="o">.</span><span class="n">append</span><span class="p">(</span><span class="n">end_time</span> <span class="o">-</span> <span class="n">start_time</span><span class="p">)</span>
            <span class="k">if</span> <span class="n">i</span><span class="o">%</span><span class="k">1000</span>==0:
                <span class="nb">print</span><span class="p">(</span><span class="s1">'Iteration </span><span class="si">%d</span><span class="s1">/</span><span class="si">%d</span><span class="s1">, ave batch time </span><span class="si">%.2f</span><span class="s1"> ms'</span><span class="o">%</span><span class="p">(</span><span class="n">i</span><span class="p">,</span> <span class="n">nruns</span><span class="p">,</span> <span class="n">np</span><span class="o">.</span><span class="n">mean</span><span class="p">(</span><span class="n">timings</span><span class="p">)</span><span class="o">*</span><span class="mi">1000</span><span class="p">))</span>

    <span class="nb">print</span><span class="p">(</span><span class="s2">"Input shape:"</span><span class="p">,</span> <span class="n">input_data</span><span class="o">.</span><span class="n">size</span><span class="p">())</span>
    <span class="nb">print</span><span class="p">(</span><span class="s2">"Output features size:"</span><span class="p">,</span> <span class="n">features</span><span class="o">.</span><span class="n">size</span><span class="p">())</span>

    <span class="nb">print</span><span class="p">(</span><span class="s1">'Average batch time: </span><span class="si">%.2f</span><span class="s1"> ms'</span><span class="o">%</span><span class="p">(</span><span class="n">np</span><span class="o">.</span><span class="n">mean</span><span class="p">(</span><span class="n">timings</span><span class="p">)</span><span class="o">*</span><span class="mi">1000</span><span class="p">))</span>
</pre>
         </div>
        </div>
       </div>
       <h3 id="PyTorch-model">
        PyTorch model
        <a class="headerlink" href="#PyTorch-model" title="Permalink to this headline">
         ¶
        </a>
       </h3>
       <div class="nbinput docutils container">
        <div class="prompt highlight-none notranslate">
         <div class="highlight">
          <pre><span></span>[4]:
</pre>
         </div>
        </div>
        <div class="input_area highlight-ipython3 notranslate">
         <div class="highlight">
          <pre>
<span></span><span class="n">model</span> <span class="o">=</span> <span class="n">LeNet</span><span class="p">()</span>
<span class="n">model</span><span class="o">.</span><span class="n">to</span><span class="p">(</span><span class="s2">"cuda"</span><span class="p">)</span><span class="o">.</span><span class="n">eval</span><span class="p">()</span>

</pre>
         </div>
        </div>
       </div>
       <div class="nboutput nblast docutils container">
        <div class="prompt highlight-none notranslate">
         <div class="highlight">
          <pre><span></span>[4]:
</pre>
         </div>
        </div>
        <div class="output_area docutils container">
         <div class="highlight">
          <pre>
LeNet(
  (feat): LeNetFeatExtractor(
    (conv1): Conv2d(1, 6, kernel_size=(3, 3), stride=(1, 1))
    (conv2): Conv2d(6, 16, kernel_size=(3, 3), stride=(1, 1))
  )
  (classifer): LeNetClassifier(
    (fc1): Linear(in_features=576, out_features=120, bias=True)
    (fc2): Linear(in_features=120, out_features=84, bias=True)
    (fc3): Linear(in_features=84, out_features=10, bias=True)
  )
)
</pre>
         </div>
        </div>
       </div>
       <div class="nbinput docutils container">
        <div class="prompt highlight-none notranslate">
         <div class="highlight">
          <pre><span></span>[5]:
</pre>
         </div>
        </div>
        <div class="input_area highlight-ipython3 notranslate">
         <div class="highlight">
          <pre>
<span></span><span class="n">benchmark</span><span class="p">(</span><span class="n">model</span><span class="p">)</span>
</pre>
         </div>
        </div>
       </div>
       <div class="nboutput nblast docutils container">
        <div class="prompt empty docutils container">
        </div>
        <div class="output_area docutils container">
         <div class="highlight">
          <pre>
Warm up ...
Start timing ...
Iteration 1000/10000, ave batch time 0.58 ms
Iteration 2000/10000, ave batch time 0.58 ms
Iteration 3000/10000, ave batch time 0.58 ms
Iteration 4000/10000, ave batch time 0.58 ms
Iteration 5000/10000, ave batch time 0.58 ms
Iteration 6000/10000, ave batch time 0.58 ms
Iteration 7000/10000, ave batch time 0.58 ms
Iteration 8000/10000, ave batch time 0.58 ms
Iteration 9000/10000, ave batch time 0.58 ms
Iteration 10000/10000, ave batch time 0.58 ms
Input shape: torch.Size([1024, 1, 32, 32])
Output features size: torch.Size([1024, 10])
Average batch time: 0.58 ms
</pre>
         </div>
        </div>
       </div>
       <p>
        When compiling your module to TorchScript, there are two paths: Tracing and Scripting.
       </p>
       <h3 id="Tracing">
        Tracing
        <a class="headerlink" href="#Tracing" title="Permalink to this headline">
         ¶
        </a>
       </h3>
       <p>
        Tracing follows the path of execution when the module is called and records what happens. This recording is what the TorchScript IR will describe. To trace an instance of our LeNet module, we can call torch.jit.trace with an example input.
       </p>
       <div class="nbinput docutils container">
        <div class="prompt highlight-none notranslate">
         <div class="highlight">
          <pre><span></span>[6]:
</pre>
         </div>
        </div>
        <div class="input_area highlight-ipython3 notranslate">
         <div class="highlight">
          <pre>
<span></span><span class="n">traced_model</span> <span class="o">=</span> <span class="n">torch</span><span class="o">.</span><span class="n">jit</span><span class="o">.</span><span class="n">trace</span><span class="p">(</span><span class="n">model</span><span class="p">,</span> <span class="n">torch</span><span class="o">.</span><span class="n">empty</span><span class="p">([</span><span class="mi">1</span><span class="p">,</span><span class="mi">1</span><span class="p">,</span><span class="mi">32</span><span class="p">,</span><span class="mi">32</span><span class="p">])</span><span class="o">.</span><span class="n">to</span><span class="p">(</span><span class="s2">"cuda"</span><span class="p">))</span>
<span class="n">traced_model</span>
</pre>
         </div>
        </div>
       </div>
       <div class="nboutput nblast docutils container">
        <div class="prompt highlight-none notranslate">
         <div class="highlight">
          <pre><span></span>[6]:
</pre>
         </div>
        </div>
        <div class="output_area docutils container">
         <div class="highlight">
          <pre>
LeNet(
  original_name=LeNet
  (feat): LeNetFeatExtractor(
    original_name=LeNetFeatExtractor
    (conv1): Conv2d(original_name=Conv2d)
    (conv2): Conv2d(original_name=Conv2d)
  )
  (classifer): LeNetClassifier(
    original_name=LeNetClassifier
    (fc1): Linear(original_name=Linear)
    (fc2): Linear(original_name=Linear)
    (fc3): Linear(original_name=Linear)
  )
)
</pre>
         </div>
        </div>
       </div>
       <div class="nbinput docutils container">
        <div class="prompt highlight-none notranslate">
         <div class="highlight">
          <pre><span></span>[7]:
</pre>
         </div>
        </div>
        <div class="input_area highlight-ipython3 notranslate">
         <div class="highlight">
          <pre>
<span></span><span class="n">benchmark</span><span class="p">(</span><span class="n">traced_model</span><span class="p">)</span>
</pre>
         </div>
        </div>
       </div>
       <div class="nboutput nblast docutils container">
        <div class="prompt empty docutils container">
        </div>
        <div class="output_area docutils container">
         <div class="highlight">
          <pre>
Warm up ...
Start timing ...
Iteration 1000/10000, ave batch time 0.58 ms
Iteration 2000/10000, ave batch time 0.59 ms
Iteration 3000/10000, ave batch time 0.59 ms
Iteration 4000/10000, ave batch time 0.59 ms
Iteration 5000/10000, ave batch time 0.59 ms
Iteration 6000/10000, ave batch time 0.59 ms
Iteration 7000/10000, ave batch time 0.59 ms
Iteration 8000/10000, ave batch time 0.59 ms
Iteration 9000/10000, ave batch time 0.59 ms
Iteration 10000/10000, ave batch time 0.59 ms
Input shape: torch.Size([1024, 1, 32, 32])
Output features size: torch.Size([1024, 10])
Average batch time: 0.59 ms
</pre>
         </div>
        </div>
       </div>
       <h3 id="Scripting">
        Scripting
        <a class="headerlink" href="#Scripting" title="Permalink to this headline">
         ¶
        </a>
       </h3>
       <p>
        Scripting actually inspects your code with a compiler and generates an equivalent TorchScript program. The difference is that since tracing simply follows the execution of your module, it cannot pick up control flow for instance, it will only follow the code path that a particular input triggers. By working from the Python code, the compiler can include these components. We can run the script compiler on our LeNet module by calling torch.jit.script.
       </p>
       <div class="nbinput nblast docutils container">
        <div class="prompt highlight-none notranslate">
         <div class="highlight">
          <pre><span></span>[8]:
</pre>
         </div>
        </div>
        <div class="input_area highlight-ipython3 notranslate">
         <div class="highlight">
          <pre>
<span></span><span class="n">model</span> <span class="o">=</span> <span class="n">LeNet</span><span class="p">()</span><span class="o">.</span><span class="n">to</span><span class="p">(</span><span class="s2">"cuda"</span><span class="p">)</span><span class="o">.</span><span class="n">eval</span><span class="p">()</span>
<span class="n">script_model</span> <span class="o">=</span> <span class="n">torch</span><span class="o">.</span><span class="n">jit</span><span class="o">.</span><span class="n">script</span><span class="p">(</span><span class="n">model</span><span class="p">)</span>

</pre>
         </div>
        </div>
       </div>
       <div class="nbinput docutils container">
        <div class="prompt highlight-none notranslate">
         <div class="highlight">
          <pre><span></span>[9]:
</pre>
         </div>
        </div>
        <div class="input_area highlight-ipython3 notranslate">
         <div class="highlight">
          <pre>
<span></span><span class="n">script_model</span>
</pre>
         </div>
        </div>
       </div>
       <div class="nboutput nblast docutils container">
        <div class="prompt highlight-none notranslate">
         <div class="highlight">
          <pre><span></span>[9]:
</pre>
         </div>
        </div>
        <div class="output_area docutils container">
         <div class="highlight">
          <pre>
RecursiveScriptModule(
  original_name=LeNet
  (feat): RecursiveScriptModule(
    original_name=LeNetFeatExtractor
    (conv1): RecursiveScriptModule(original_name=Conv2d)
    (conv2): RecursiveScriptModule(original_name=Conv2d)
  )
  (classifer): RecursiveScriptModule(
    original_name=LeNetClassifier
    (fc1): RecursiveScriptModule(original_name=Linear)
    (fc2): RecursiveScriptModule(original_name=Linear)
    (fc3): RecursiveScriptModule(original_name=Linear)
  )
)
</pre>
         </div>
        </div>
       </div>
       <div class="nbinput docutils container">
        <div class="prompt highlight-none notranslate">
         <div class="highlight">
          <pre><span></span>[10]:
</pre>
         </div>
        </div>
        <div class="input_area highlight-ipython3 notranslate">
         <div class="highlight">
          <pre>
<span></span><span class="n">benchmark</span><span class="p">(</span><span class="n">script_model</span><span class="p">)</span>
</pre>
         </div>
        </div>
       </div>
       <div class="nboutput nblast docutils container">
        <div class="prompt empty docutils container">
        </div>
        <div class="output_area docutils container">
         <div class="highlight">
          <pre>
Warm up ...
Start timing ...
Iteration 1000/10000, ave batch time 0.59 ms
Iteration 2000/10000, ave batch time 0.59 ms
Iteration 3000/10000, ave batch time 0.59 ms
Iteration 4000/10000, ave batch time 0.59 ms
Iteration 5000/10000, ave batch time 0.59 ms
Iteration 6000/10000, ave batch time 0.59 ms
Iteration 7000/10000, ave batch time 0.59 ms
Iteration 8000/10000, ave batch time 0.59 ms
Iteration 9000/10000, ave batch time 0.59 ms
Iteration 10000/10000, ave batch time 0.59 ms
Input shape: torch.Size([1024, 1, 32, 32])
Output features size: torch.Size([1024, 10])
Average batch time: 0.59 ms
</pre>
         </div>
        </div>
       </div>
       <p>
        ## 3. Compiling with TRTorch
       </p>
       <h3 id="TorchScript-traced-model">
        TorchScript traced model
        <a class="headerlink" href="#TorchScript-traced-model" title="Permalink to this headline">
         ¶
        </a>
       </h3>
       <p>
        First, we compile the TorchScript traced model with TRTorch. Notice the performance impact.
       </p>
       <div class="nbinput nblast docutils container">
        <div class="prompt highlight-none notranslate">
         <div class="highlight">
          <pre><span></span>[11]:
</pre>
         </div>
        </div>
        <div class="input_area highlight-ipython3 notranslate">
         <div class="highlight">
          <pre>
<span></span><span class="kn">import</span> <span class="nn">trtorch</span>

<span class="c1"># We use a batch-size of 1024, and half precision</span>
<span class="n">compile_settings</span> <span class="o">=</span> <span class="p">{</span>
    <span class="s2">"input_shapes"</span><span class="p">:</span> <span class="p">[</span>
        <span class="p">{</span>
            <span class="s2">"min"</span> <span class="p">:</span> <span class="p">[</span><span class="mi">1024</span><span class="p">,</span> <span class="mi">1</span><span class="p">,</span> <span class="mi">32</span><span class="p">,</span> <span class="mi">32</span><span class="p">],</span>
            <span class="s2">"opt"</span> <span class="p">:</span> <span class="p">[</span><span class="mi">1024</span><span class="p">,</span> <span class="mi">1</span><span class="p">,</span> <span class="mi">33</span><span class="p">,</span> <span class="mi">33</span><span class="p">],</span>
            <span class="s2">"max"</span> <span class="p">:</span> <span class="p">[</span><span class="mi">1024</span><span class="p">,</span> <span class="mi">1</span><span class="p">,</span> <span class="mi">34</span><span class="p">,</span> <span class="mi">34</span><span class="p">],</span>
        <span class="p">}</span>
    <span class="p">],</span>
    <span class="s2">"op_precision"</span><span class="p">:</span> <span class="n">torch</span><span class="o">.</span><span class="n">half</span> <span class="c1"># Run with FP16</span>
<span class="p">}</span>

<span class="n">trt_ts_module</span> <span class="o">=</span> <span class="n">trtorch</span><span class="o">.</span><span class="n">compile</span><span class="p">(</span><span class="n">traced_model</span><span class="p">,</span> <span class="n">compile_settings</span><span class="p">)</span>

<span class="n">input_data</span> <span class="o">=</span> <span class="n">torch</span><span class="o">.</span><span class="n">randn</span><span class="p">((</span><span class="mi">1024</span><span class="p">,</span> <span class="mi">1</span><span class="p">,</span> <span class="mi">32</span><span class="p">,</span> <span class="mi">32</span><span class="p">))</span>
<span class="n">input_data</span> <span class="o">=</span> <span class="n">input_data</span><span class="o">.</span><span class="n">half</span><span class="p">()</span><span class="o">.</span><span class="n">to</span><span class="p">(</span><span class="s2">"cuda"</span><span class="p">)</span>

<span class="n">input_data</span> <span class="o">=</span> <span class="n">input_data</span><span class="o">.</span><span class="n">half</span><span class="p">()</span>
<span class="n">result</span> <span class="o">=</span> <span class="n">trt_ts_module</span><span class="p">(</span><span class="n">input_data</span><span class="p">)</span>
<span class="n">torch</span><span class="o">.</span><span class="n">jit</span><span class="o">.</span><span class="n">save</span><span class="p">(</span><span class="n">trt_ts_module</span><span class="p">,</span> <span class="s2">"trt_ts_module.ts"</span><span class="p">)</span>
</pre>
         </div>
        </div>
       </div>
       <div class="nbinput docutils container">
        <div class="prompt highlight-none notranslate">
         <div class="highlight">
          <pre><span></span>[12]:
</pre>
         </div>
        </div>
        <div class="input_area highlight-ipython3 notranslate">
         <div class="highlight">
          <pre>
<span></span><span class="n">benchmark</span><span class="p">(</span><span class="n">trt_ts_module</span><span class="p">,</span> <span class="n">input_shape</span><span class="o">=</span><span class="p">(</span><span class="mi">1024</span><span class="p">,</span> <span class="mi">1</span><span class="p">,</span> <span class="mi">32</span><span class="p">,</span> <span class="mi">32</span><span class="p">),</span> <span class="n">dtype</span><span class="o">=</span><span class="s2">"fp16"</span><span class="p">)</span>
</pre>
         </div>
        </div>
       </div>
       <div class="nboutput nblast docutils container">
        <div class="prompt empty docutils container">
        </div>
        <div class="output_area docutils container">
         <div class="highlight">
          <pre>
Warm up ...
Start timing ...
Iteration 1000/10000, ave batch time 0.41 ms
Iteration 2000/10000, ave batch time 0.41 ms
Iteration 3000/10000, ave batch time 0.41 ms
Iteration 4000/10000, ave batch time 0.40 ms
Iteration 5000/10000, ave batch time 0.40 ms
Iteration 6000/10000, ave batch time 0.40 ms
Iteration 7000/10000, ave batch time 0.40 ms
Iteration 8000/10000, ave batch time 0.40 ms
Iteration 9000/10000, ave batch time 0.40 ms
Iteration 10000/10000, ave batch time 0.40 ms
Input shape: torch.Size([1024, 1, 32, 32])
Output features size: torch.Size([1024, 10])
Average batch time: 0.40 ms
</pre>
         </div>
        </div>
       </div>
       <h3 id="TorchScript-script-model">
        TorchScript script model
        <a class="headerlink" href="#TorchScript-script-model" title="Permalink to this headline">
         ¶
        </a>
       </h3>
       <p>
        Next, we compile the TorchScript script model with TRTorch. Notice the performance impact.
       </p>
       <div class="nbinput nblast docutils container">
        <div class="prompt highlight-none notranslate">
         <div class="highlight">
          <pre><span></span>[14]:
</pre>
         </div>
        </div>
        <div class="input_area highlight-ipython3 notranslate">
         <div class="highlight">
          <pre>
<span></span><span class="kn">import</span> <span class="nn">trtorch</span>

<span class="c1"># We use a batch-size of 1024, and half precision</span>
<span class="n">compile_settings</span> <span class="o">=</span> <span class="p">{</span>
    <span class="s2">"input_shapes"</span><span class="p">:</span> <span class="p">[</span>
        <span class="p">{</span>
            <span class="s2">"min"</span> <span class="p">:</span> <span class="p">[</span><span class="mi">1024</span><span class="p">,</span> <span class="mi">1</span><span class="p">,</span> <span class="mi">32</span><span class="p">,</span> <span class="mi">32</span><span class="p">],</span>
            <span class="s2">"opt"</span> <span class="p">:</span> <span class="p">[</span><span class="mi">1024</span><span class="p">,</span> <span class="mi">1</span><span class="p">,</span> <span class="mi">33</span><span class="p">,</span> <span class="mi">33</span><span class="p">],</span>
            <span class="s2">"max"</span> <span class="p">:</span> <span class="p">[</span><span class="mi">1024</span><span class="p">,</span> <span class="mi">1</span><span class="p">,</span> <span class="mi">34</span><span class="p">,</span> <span class="mi">34</span><span class="p">],</span>
        <span class="p">}</span>
    <span class="p">],</span>
    <span class="s2">"op_precision"</span><span class="p">:</span> <span class="n">torch</span><span class="o">.</span><span class="n">half</span> <span class="c1"># Run with FP16</span>
<span class="p">}</span>

<span class="n">trt_script_module</span> <span class="o">=</span> <span class="n">trtorch</span><span class="o">.</span><span class="n">compile</span><span class="p">(</span><span class="n">script_model</span><span class="p">,</span> <span class="n">compile_settings</span><span class="p">)</span>

<span class="n">input_data</span> <span class="o">=</span> <span class="n">torch</span><span class="o">.</span><span class="n">randn</span><span class="p">((</span><span class="mi">1024</span><span class="p">,</span> <span class="mi">1</span><span class="p">,</span> <span class="mi">32</span><span class="p">,</span> <span class="mi">32</span><span class="p">))</span>
<span class="n">input_data</span> <span class="o">=</span> <span class="n">input_data</span><span class="o">.</span><span class="n">half</span><span class="p">()</span><span class="o">.</span><span class="n">to</span><span class="p">(</span><span class="s2">"cuda"</span><span class="p">)</span>

<span class="n">input_data</span> <span class="o">=</span> <span class="n">input_data</span><span class="o">.</span><span class="n">half</span><span class="p">()</span>
<span class="n">result</span> <span class="o">=</span> <span class="n">trt_script_module</span><span class="p">(</span><span class="n">input_data</span><span class="p">)</span>
<span class="n">torch</span><span class="o">.</span><span class="n">jit</span><span class="o">.</span><span class="n">save</span><span class="p">(</span><span class="n">trt_script_module</span><span class="p">,</span> <span class="s2">"trt_script_module.ts"</span><span class="p">)</span>
</pre>
         </div>
        </div>
       </div>
       <div class="nbinput docutils container">
        <div class="prompt highlight-none notranslate">
         <div class="highlight">
          <pre><span></span>[15]:
</pre>
         </div>
        </div>
        <div class="input_area highlight-ipython3 notranslate">
         <div class="highlight">
          <pre>
<span></span><span class="n">benchmark</span><span class="p">(</span><span class="n">trt_script_module</span><span class="p">,</span> <span class="n">input_shape</span><span class="o">=</span><span class="p">(</span><span class="mi">1024</span><span class="p">,</span> <span class="mi">1</span><span class="p">,</span> <span class="mi">32</span><span class="p">,</span> <span class="mi">32</span><span class="p">),</span> <span class="n">dtype</span><span class="o">=</span><span class="s2">"fp16"</span><span class="p">)</span>
</pre>
         </div>
        </div>
       </div>
       <div class="nboutput nblast docutils container">
        <div class="prompt empty docutils container">
        </div>
        <div class="output_area docutils container">
         <div class="highlight">
          <pre>
Warm up ...
Start timing ...
Iteration 1000/10000, ave batch time 0.40 ms
Iteration 2000/10000, ave batch time 0.40 ms
Iteration 3000/10000, ave batch time 0.40 ms
Iteration 4000/10000, ave batch time 0.40 ms
Iteration 5000/10000, ave batch time 0.40 ms
Iteration 6000/10000, ave batch time 0.40 ms
Iteration 7000/10000, ave batch time 0.40 ms
Iteration 8000/10000, ave batch time 0.40 ms
Iteration 9000/10000, ave batch time 0.40 ms
Iteration 10000/10000, ave batch time 0.40 ms
Input shape: torch.Size([1024, 1, 32, 32])
Output features size: torch.Size([1024, 10])
Average batch time: 0.40 ms
</pre>
         </div>
        </div>
       </div>
       <h2 id="Conclusion">
        Conclusion
        <a class="headerlink" href="#Conclusion" title="Permalink to this headline">
         ¶
        </a>
       </h2>
       <p>
        In this notebook, we have walked through the complete process of compiling TorchScript models with TRTorch and test the performance impact of the optimization.
       </p>
       <h3 id="What’s-next">
        What’s next
        <a class="headerlink" href="#What’s-next" title="Permalink to this headline">
         ¶
        </a>
       </h3>
       <p>
        Now it’s time to try TRTorch on your own model. Fill out issues at
        <a class="reference external" href="https://github.com/NVIDIA/TRTorch">
         https://github.com/NVIDIA/TRTorch
        </a>
        . Your involvement will help future development of TRTorch.
       </p>
      </article>
     </div>
    </div>
   </main>
  </div>
  <footer class="md-footer">
   <div class="md-footer-nav">
    <nav class="md-footer-nav__inner md-grid">
     <a class="md-flex md-footer-nav__link md-footer-nav__link--prev" href="../tutorials/use_from_pytorch.html" rel="prev" title="Using TRTorch Directly From PyTorch">
      <div class="md-flex__cell md-flex__cell--shrink">
       <i class="md-icon md-icon--arrow-back md-footer-nav__button">
       </i>
      </div>
      <div class="md-flex__cell md-flex__cell--stretch md-footer-nav__title">
       <span class="md-flex__ellipsis">
        <span class="md-footer-nav__direction">
         Previous
        </span>
        Using TRTorch Directly From PyTorch
       </span>
      </div>
     </a>
     <a class="md-flex md-footer-nav__link md-footer-nav__link--next" href="ssd-object-detection-demo.html" rel="next" title="Object Detection with TRTorch (SSD)">
      <div class="md-flex__cell md-flex__cell--stretch md-footer-nav__title">
       <span class="md-flex__ellipsis">
        <span class="md-footer-nav__direction">
         Next
        </span>
        Object Detection with TRTorch (SSD)
       </span>
      </div>
      <div class="md-flex__cell md-flex__cell--shrink">
       <i class="md-icon md-icon--arrow-forward md-footer-nav__button">
       </i>
      </div>
     </a>
    </nav>
   </div>
   <div class="md-footer-meta md-typeset">
    <div class="md-footer-meta__inner md-grid">
     <div class="md-footer-copyright">
      <div class="md-footer-copyright__highlight">
       © Copyright 2020, NVIDIA Corporation.
      </div>
      Created using
      <a href="http://www.sphinx-doc.org/">
       Sphinx
      </a>
      3.1.2.
             and
      <a href="https://github.com/bashtage/sphinx-material/">
       Material for
              Sphinx
      </a>
     </div>
    </div>
   </div>
  </footer>
  <script src="../_static/javascripts/application.js">
  </script>
  <script>
   app.initialize({version: "1.0.4", url: {base: ".."}})
  </script>
 </body>
</html>